

egrep "^\(.*\),$" Megatable-2014.sql > mt-2014.clean.csv
egrep "^\(.*\),$" Megatable-2015.sql > mt-2015.clean.csv
egrep "^\(.*\),$" Megatable-2016.sql > mt-2016.clean.csv

cat mt-2014.clean.csv | cut -d"," -f1,2,3,8,9 > mt-2014.slice.csv
cat mt-2015.clean.csv | cut -d"," -f1,2,3,8,9 > mt-2015.slice.csv
cat mt-2016.clean.csv | cut -d"," -f1,2,3,8,9 > mt-2016.slice.csv


(1266, 'http://zeenews.india.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1267, 'http://zeenews.india.com/sports/index.html', 'sport', '0000-00-00 00:00:00', -1),
(1268, 'http://zeenews.india.com/business/index.html', 'business', '0000-00-00 00:00:00', -1),
(1269, 'http://zeenews.india.com/sci-tech/index.html', 'technology', '0000-00-00 00:00:00', -1),
(1270, 'http://zeenews.india.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),

cat mt-*clean.csv | grep "1266)," > mt-zeeindia-extract-1266.csv
cat mt-*clean.csv | grep "1267)," > mt-zeeindia-extract-1267.csv
cat mt-*clean.csv | grep "1268)," > mt-zeeindia-extract-1268.csv
cat mt-*clean.csv | grep "1269)," > mt-zeeindia-extract-1269.csv
cat mt-*clean.csv | grep "1270)," > mt-zeeindia-extract-1270.csv



CREATE TABLE IF NOT EXISTS `Megatable` (
  `id` double NOT NULL DEFAULT '0',
  `newsHeadline` text CHARACTER SET utf8 NOT NULL,
  `start_time_stamp` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
  `end_time_stamp` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
  `first_occurance` int(11) NOT NULL DEFAULT '0', //useless
  `last_occurance` int(11) NOT NULL DEFAULT '0', //useless
  `total_count` int(11) NOT NULL DEFAULT '0',
  `URL` varchar(512) CHARACTER SET utf8 NOT NULL DEFAULT '',
  `source_id` int(11) NOT NULL DEFAULT '0'
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

--
-- Dumping data for table `Megatable`
--

INSERT INTO `Megatable` (`id`, `newsHeadline`, `start_time_stamp`, `end_time_stamp`, `first_occurance`, `last_occurance`, `total_count`, `URL`, `source_id`) VALUES

INSERT INTO `source_info` (`id`, `URL`, `category`, `last_acess_time`, `status`) VALUES
(1222, 'http://indiatoday.intoday.in/', 'homepage', '0000-00-00 00:00:00', -1),
(1223, 'http://indiatoday.intoday.in/section/84/1/sports.html?source=homesptn', 'sport', '0000-00-00 00:00:00', -1),
(1224, 'http://businesstoday.intoday.in/', 'business', '0000-00-00 00:00:00', -1),
(1225, 'http://indiatoday.intoday.in/section/229/1/technology.html', 'technology', '0000-00-00 00:00:00', -1),
(1226, 'http://indiatoday.intoday.in/section/67/1/movies.html?source=homemoviestn', 'entertainment', '0000-00-00 00:00:00', -1),
(1227, 'http://indianexpress.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1228, 'http://indianexpress.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1229, 'http://indianexpress.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1230, 'http://indianexpress.com/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1231, 'http://indianexpress.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),
(1232, 'http://timesofindia.indiatimes.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1233, 'http://timesofindia.indiatimes.com/sports', 'sport', '0000-00-00 00:00:00', -1),
(1234, 'http://timesofindia.indiatimes.com/business', 'business', '0000-00-00 00:00:00', -1),
(1235, 'http://timesofindia.indiatimes.com/tech', 'technology', '0000-00-00 00:00:00', -1),
(1236, 'http://timesofindia.indiatimes.com/entertainment', 'entertainment', '0000-00-00 00:00:00', -1),
(1237, 'http://www.thehindu.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1238, 'http://www.thehindu.com/sport/', 'sport', '0000-00-00 00:00:00', -1),
(1239, 'http://www.thehindu.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1240, 'http://www.thehindu.com/sci-tech/', 'technology', '0000-00-00 00:00:00', -1),
(1241, 'http://www.hindustantimes.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1242, 'http://www.hindustantimes.com/sports-news/', 'sport', '0000-00-00 00:00:00', -1),
(1243, 'http://www.hindustantimes.com/business-news/', 'business', '0000-00-00 00:00:00', -1),
(1244, 'http://www.hindustantimes.com/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1245, 'http://www.hindustantimes.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),
(1246, 'http://edition.cnn.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1247, 'http://edition.cnn.com/SPORT/', 'sport', '0000-00-00 00:00:00', -1),
(1248, 'http://edition.cnn.com/BUSINESS/', 'business', '0000-00-00 00:00:00', -1),
(1249, 'http://edition.cnn.com/TECH/', 'technology', '0000-00-00 00:00:00', -1),
(1250, 'http://edition.cnn.com/SHOWBIZ/', 'entertainment', '0000-00-00 00:00:00', -1),
(1251, 'http://www.usatoday.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1252, 'http://www.usatoday.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1253, 'http://www.usatoday.com/money/', 'business', '0000-00-00 00:00:00', -1),
(1254, 'http://www.usatoday.com/tech/', 'technology', '0000-00-00 00:00:00', -1),
(1255, 'http://www.usatoday.com/life/', 'entertainment', '0000-00-00 00:00:00', -1),
(1256, 'http://www.bbc.com/news/', 'homepage', '0000-00-00 00:00:00', -1),
(1257, 'http://www.bbc.com/sport/0/', 'soprt', '0000-00-00 00:00:00', -1),
(1258, 'http://www.bbc.com/news/business/', 'business', '0000-00-00 00:00:00', -1),
(1259, 'http://www.bbc.com/news/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1260, 'http://www.bbc.com/news/entertainment_and_arts/', 'entertainment', '0000-00-00 00:00:00', -1),
(1261, 'http://www.ndtv.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1262, 'http://sports.ndtv.com/', 'sport', '0000-00-00 00:00:00', -1),
(1263, 'http://profit.ndtv.com/', 'business', '0000-00-00 00:00:00', -1),
(1264, 'http://gadgets.ndtv.com/', 'technology', '0000-00-00 00:00:00', -1),
(1265, 'http://movies.ndtv.com/', 'entertainment', '0000-00-00 00:00:00', -1),
(1266, 'http://zeenews.india.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1267, 'http://zeenews.india.com/sports/index.html', 'sport', '0000-00-00 00:00:00', -1),
(1268, 'http://zeenews.india.com/business/index.html', 'business', '0000-00-00 00:00:00', -1),
(1269, 'http://zeenews.india.com/sci-tech/index.html', 'technology', '0000-00-00 00:00:00', -1),
(1270, 'http://zeenews.india.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),
(1271, 'http://ibnlive.in.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1272, 'http://ibnlive.in.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1273, 'http://ibnlive.in.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1274, 'http://ibnlive.in.com/tech/', 'technology', '0000-00-00 00:00:00', -1),
(1275, 'http://ibnlive.in.com/movies/', 'entertainment', '0000-00-00 00:00:00', -1),
(1276, 'http://abcnews.go.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1277, 'http://abcnews.go.com/sports', 'sport', '0000-00-00 00:00:00', -1),
(1278, 'http://abcnews.go.com/business', 'business', '0000-00-00 00:00:00', -1),
(1279, 'http://abcnews.go.com/technology', 'techology', '0000-00-00 00:00:00', -1),
(1280, 'http://abcnews.go.com/entertainment', 'entertainment', '0000-00-00 00:00:00', -1),
(1281, 'http://24x7reporting.in/', 'homepage', '0000-00-00 00:00:00', -1),
(1282, 'http://24x7reporting.in/sports', 'sport', '0000-00-00 00:00:00', -1),
(1283, 'http://24x7reporting.in/business', 'business', '0000-00-00 00:00:00', -1),
(1284, 'http://24x7reporting.in/technology', 'technology', '0000-00-00 00:00:00', -1),
(1285, 'http://24x7reporting.in/entertainment', 'entertainment', '0000-00-00 00:00:00', -1),
(1286, 'http://www.nbcnews.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1287, 'http://nbcsports.nbcnews.com/', 'sport', '0000-00-00 00:00:00', -1),
(1288, 'http://www.nbcnews.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1289, 'http://www.nbcnews.com/technology', 'technology', '0000-00-00 00:00:00', -1),
(1290, 'http://www.today.com/entertainment', 'entertainment', '0000-00-00 00:00:00', -1),
(1291, 'http://www.washingtonpost.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1292, 'http://www.washingtonpost.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1293, 'http://www.washingtonpost.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1294, 'http://www.washingtonpost.com/business/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1295, 'http://www.washingtonpost.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),
(1296, 'http://www.chicagotribune.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1297, 'http://www.chicagotribune.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1298, 'http://www.chicagotribune.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1299, 'http://www.chicagotribune.com/business/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1300, 'http://www.chicagotribune.com/entertainment/', 'entertainment', '0000-00-00 00:00:00', -1),
(1301, 'http://www.suntimes.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1302, 'http://www.suntimes.com/sports/', 'sport', '0000-00-00 00:00:00', -1),
(1303, 'http://www.suntimes.com/business/index.html', 'business', '0000-00-00 00:00:00', -1),
(1304, 'http://www.suntimes.com/entertainment/index.html', 'entertainment', '0000-00-00 00:00:00', -1),
(1305, 'http://www.cnbc.com/id/100746255?region=world', 'homepage', '0000-00-00 00:00:00', -1),
(1306, 'http://www.cnbc.com/id/10000664', 'business', '0000-00-00 00:00:00', -1),
(1307, 'http://www.cnbc.com/id/19854910', 'technology', '0000-00-00 00:00:00', -1),
(1308, 'http://www.tribuneindia.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1309, 'http://www.adn.com/', 'Homepage', '0000-00-00 00:00:00', -1),
(1310, 'http://www.adn.com/section/news', 'news', '0000-00-00 00:00:00', -1),
(1311, 'http://www.adn.com/section/politics', 'politics', '0000-00-00 00:00:00', -1),
(1312, 'http://www.adn.com/section/sports', 'sports', '0000-00-00 00:00:00', -1),
(1313, 'http://www.adn.com/voices', 'commentry', '0000-00-00 00:00:00', -1),
(1314, 'http://www.adn.com/term/arctic', 'arctic', '0000-00-00 00:00:00', -1),
(1315, 'http://www.adn.com/section/culture', 'culture', '0000-00-00 00:00:00', -1),
(1316, 'http://www.adn.com/multimedia', 'multimedia', '0000-00-00 00:00:00', -1),
(1317, 'http://www.huffingtonpost.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1318, 'http://www.huffingtonpost.com/business/', 'business', '0000-00-00 00:00:00', -1),
(1319, 'http://www.huffingtonpost.com/sports/', 'sports', '0000-00-00 00:00:00', -1),
(1320, 'http://www.huffingtonpost.com/Entertainment/', 'Entertainment', '0000-00-00 00:00:00', -1),
(1321, 'http://www.huffingtonpost.com/media/', 'media', '0000-00-00 00:00:00', -1),
(1322, 'http://www.huffingtonpost.com/theworldpost/', 'theworldpost', '0000-00-00 00:00:00', -1),
(1323, 'http://www.huffingtonpost.com/green/', 'green', '0000-00-00 00:00:00', -1),
(1324, 'http://www.huffingtonpost.com/technology/', 'technology', '0000-00-00 00:00:00', -1),
(1325, 'http://techcrunch.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1326, 'http://techcrunch.com/event/', 'event', '0000-00-00 00:00:00', -1),
(1327, 'http://techcrunch.com/popular/', 'popular', '0000-00-00 00:00:00', -1),
(1328, 'http://www.businessworld.in/', 'homepage', '0000-00-00 00:00:00', -1),
(1329, 'http://www.businessworld.in/economy', 'economy', '0000-00-00 00:00:00', -1),
(1331, 'http://www.businessworld.in/b-school', 'b-school', '0000-00-00 00:00:00', -1),
(1332, 'http://www.businessworld.in/science-and-technology', 'technology', '0000-00-00 00:00:00', -1),
(1333, 'http://www.financialexpress.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1334, 'http://www.financialexpress.com/section/tech/377', 'technology', '0000-00-00 00:00:00', -1),
(1335, 'http://www.financialexpress.com/section/economy/97', 'economy', '0000-00-00 00:00:00', -1),
(1336, 'http://www.financialexpress.com/section/corporate-news/95', 'corporate', '0000-00-00 00:00:00', -1),
(1337, 'http://www.financialexpress.com/section/industry/1294', 'Industry', '0000-00-00 00:00:00', -1),
(1338, 'http://www.thehindubusinessline.com/markets/', 'Markets', '0000-00-00 00:00:00', -1),
(1339, 'http://www.thehindubusinessline.com/features/smartbuy/', 'shopping', '0000-00-00 00:00:00', -1),
(1340, 'http://www.thehindubusinessline.com/economy/', 'economy', '0000-00-00 00:00:00', -1),
(1341, 'http://www.thehindubusinessline.com/opinion/', 'opinion', '0000-00-00 00:00:00', -1),
(1342, 'http://www.bollywoodhungama.com/', 'homepage', '0000-00-00 00:00:00', -1),
(1343, 'http://www.bollywoodhungama.com/movies', 'movies', '0000-00-00 00:00:00', -1),
(1344, 'http://www.bollywoodhungama.com/celebrities', 'celebrities', '0000-00-00 00:00:00', -1),
(1345, 'http://www.bollywoodhungama.com/more/music', 'music', '0000-00-00 00:00:00', -1),
(1346, 'http://www.bollywoodhungama.com/more/videos', 'videos', '0000-00-00 00:00:00', -1),
(1501, 'https://in.finance.yahoo.com/news/', 'business', '0000-00-00 00:00:00', -1),
(1502, 'http://money.rediff.com/news', 'business', '0000-00-00 00:00:00', -1),
(1503, 'http://www.indiainfoline.com/', 'business', '0000-00-00 00:00:00', -1),
(1504, 'http://in.reuters.com/finance/markets/india-stock-market', 'business', '0000-00-00 00:00:00', -1),
(1505, 'http://www.moneycontrol.com/', 'business', '0000-00-00 00:00:00', -1),
(1506, 'http://economictimes.indiatimes.com/news/economy/finance', 'business', '0000-00-00 00:00:00', -1),
(1507, 'http://www.thehindubusinessline.com/', 'business', '0000-00-00 00:00:00', -1),
(1508, 'http://www.financialexpress.com/section/markets/96', 'business', '0000-00-00 00:00:00', -1),
(1509, 'http://www.financialexpress.com/', 'main', '0000-00-00 00:00:00', -1),
(1510, 'http://www.business-standard.com/', 'business', '0000-00-00 00:00:00', -1),
(1511, 'http://www.livemint.com/', 'business', '0000-00-00 00:00:00', -1),
(1512, 'http://www.mydigitalfc.com/', 'business', '0000-00-00 00:00:00', -1),
(1513, 'http://www.ft.com/home/india', 'business', '0000-00-00 00:00:00', -1),



(1601, 'http://timesofindia.indiatimes.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1602, 'http://www.ibnlive.com/videos/', 'Video', '0000-00-00 00:00:00', -1),
(1603, 'http://video.ft.com', 'Video', '0000-00-00 00:00:00', -1),
(1604, 'http://www.mydigitalfc.com/fc-video-gallery', 'Video', '0000-00-00 00:00:00', -1),
(1605, 'http://www.livemint.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1606, 'http://www.business-standard.com/general-video/list', 'Video', '0000-00-00 00:00:00', -1),
(1607, 'http://www.thehindubusinessline.com/multimedia/video', 'Video', '0000-00-00 00:00:00', -1),
(1608, 'http://www.moneycontrol.com/tv/listing/latest_videos', 'Video', '0000-00-00 00:00:00', -1),
(1609, 'http://in.reuters.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1610, 'https://in.finance.yahoo.com/video/', 'Video', '0000-00-00 00:00:00', -1),
(1611, 'http://www.bollywoodhungama.com/more/videos', 'Video', '0000-00-00 00:00:00', -1),
(1612, 'http://www.financialexpress.com/videos/', 'Video', '0000-00-00 00:00:00', -1),
(1613, 'http://www.businessworld.in/category/Video-110/?page=1', 'Video', '0000-00-00 00:00:00', -1),
(1614, 'http://techcrunch.com/video/', 'Video', '0000-00-00 00:00:00', -1),
(1615, 'http://www.adn.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1616, 'http://www.tribuneindia.com/video/team-india-gears-up-for-2nd-odi-in-brisbane/13605.html', 'Video', '0000-00-00 00:00:00', -1),
(1617, 'http://video.cnbc.com/gallery', 'Video', '0000-00-00 00:00:00', -1),
(1618, 'http://www.chicagotribune.com/sports/video', 'Video', '0000-00-00 00:00:00', -1),
(1619, 'https://www.washingtonpost.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1620, 'http://www.today.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1621, 'http://www.nbcnews.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1622, 'http://www.nbcsports.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1623, 'http://abcnews.go.com/Video', 'Video', '0000-00-00 00:00:00', -1),
(1624, 'http://www.ibnlive.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1625, 'http://zeenews.india.com/videos/default.html', 'Video', '0000-00-00 00:00:00', -1),
(1626, 'http://movies.ndtv.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1627, 'http://gadgets.ndtv.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1628, 'http://profit.ndtv.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1629, 'http://sports.ndtv.com/athletics/videos', 'Video', '0000-00-00 00:00:00', -1),
(1630, 'http://www.ndtv.com/video', 'Video', '0000-00-00 00:00:00', -1),
(1631, 'http://www.bbc.com/news/video_and_audio', 'Video', '0000-00-00 00:00:00', -1),
(1632, 'http://www.usatoday.com/media/latest/videos/news', 'Video', '0000-00-00 00:00:00', -1),
(1633, 'http://edition.cnn.com/videos', 'Video', '0000-00-00 00:00:00', -1),
(1634, 'http://www.thehindu.com/videos/', 'Video', '0000-00-00 00:00:00', -1),
(1635, 'http://www.hindustantimes.com/videos/', 'Video', '0000-00-00 00:00:00', -1),
(1636, 'http://indianexpress.com/videos/', 'Video', '0000-00-00 00:00:00', -1),
(1637, 'http://indiatoday.intoday.in/videolist', 'Video', '0000-00-00 00:00:00', -1),
(1638, 'http://www.businesstoday.in/videos', 'Video', '0000-00-00 00:00:00', -1);